import re

import lxml.html
import requests
from lxml import etree


def spider_index():
    s = requests.session()
    t = s.get("https://www.brunel.ac.uk/").text

    etr = etree.HTML(t)

    a_like = etr.xpath("//*[@id='mainNav']/ul/li[2]/div/div/div")

    for text in a_like:
        dom_text = lxml.html.tostring(text)
        etr_text = etree.HTML(dom_text)
        li = etr_text.xpath("//ul/li[@class='mainNav__level2']")
        for lt in li:
            base = "https://www.brunel.ac.uk"
            dom_l = lxml.html.tostring(lt)
            etr_l = etree.HTML(dom_l)
            url1 = etr_l.xpath("//a/@href")[0]
            academy = etr_l.xpath("//a/text()")[0]
            academy_url = base + url1
            get_next("布鲁内尔大学", "https://www.brunel.ac.uk/", academy_url, academy)


def get_next(school, school_url, academy_url, academy):
    s = requests.session()
    html = s.get(academy_url).text
    etr = etree.HTML(html)
    urls = etr.xpath('//a')
    yz_url = ""
    for u in urls:
        yz = u.xpath("string()")
        if "Staff profiles" in yz or "Our staff" in yz:
            yz_url = u.xpath("@href")
    if len(yz_url) > 0:
        if "http" in yz_url[0]:
            list_url = yz_url[0]
            get_next1(school, school_url, academy_url, academy, list_url)


def get_next1(school, school_url, academy_url, academy, list_url):
    print(school, school_url, academy_url, academy, list_url)
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/91.0.4472.114 Safari/537.36 ",

    }

    # print(list_url)
    s1 = requests.session()
    s1.headers = header
    html = s1.get(list_url).content
    html = str(html)[2:-1].replace(r"\r", "").replace(r"\n", "")
    # tr = re.findall(b'<table class="table">(.*?)</table>', html)
    # print(html)
    etr = etree.HTML(html)
    institute = etr.xpath("//h1/text()")[0]
    # print(institute)
    tr = etr.xpath("//tr")
    for t in tr:
        dom_t = lxml.html.tostring(t)
        dom_t = str(dom_t)[2:-1]
        etr_t = etree.HTML(dom_t)
        # print(dom_t)
        td = etr_t.xpath("//td")

        header_img = ''
        mainurl = ''
        name = ''
        phone = ''
        email = ''
        xh_index = 1
        for dom_td in td:
            dom_tt = lxml.html.tostring(dom_td)
            # 单个td 的dom 节点解析
            dom_ttt = str(dom_tt)[2:-1]
            etr_t_dom = etree.HTML(dom_ttt)
            # 图片
            if xh_index == 1:
                img = etr_t_dom.xpath("//img/@data-src")[0]
                header_img = "https://www.brunel.ac.uk" + img

            # name
            if xh_index == 2:
                mainurl = "https://www.brunel.ac.uk" + etr_t_dom.xpath("//a/@href")[0]
                name = etr_t_dom.xpath("//a/text()")[0]

            # 电话
            if xh_index == 3:
                phone = etr_t_dom.xpath("//td/text()")[0]

            # 邮箱
            if xh_index == 4:
                email = etr_t_dom.xpath("//td/text()")[0]

            # 住址
            if xh_index == 5:
                pass
                # office = etr_t_dom.xpath("//td/text()")[0]
            xh_index += 1
        print(school, school_url, academy_url, academy, list_url, institute, header_img, name, mainurl, phone, email)


if __name__ == '__main__':
    spider_index()
